from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import json
NEWS_TAB="//div[@class='mb10 tw3_01_2']"
NEWS_URLTITLE=".//span[@class='tw3_01_2_t']//a"
NEWS_TIME=".//span[@class='tw3_01_2_t']//b"
NEWS_CONTENT=".//div[@id='Content']//p"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(options=chrome_options)
driver2 = webdriver.Chrome(options=chrome_options)
number=9
news_dict_list = []
for i in range(20,number+1):    
    url=f'https://www.chinadaily.com.cn/china/governmentandpolicy/page_{i}.html'
    driver.get(url)
    tab_list=driver.find_elements(By.XPATH,NEWS_TAB)
    
    for tab in tab_list:
        news_urltitle = tab.find_element(By.XPATH,NEWS_URLTITLE)
        news_url = news_urltitle.get_attribute("href")
        news_title = news_urltitle.text
        news_time = tab.find_element(By.XPATH,NEWS_TIME).text
        driver2.get(news_url) 
        news_content_split=driver2.find_elements(By.XPATH,NEWS_CONTENT)
        news_content = ''
        for content in news_content_split:
            news_content+=content.text
        news = {
            "title":news_title,
            "time":news_time,
            "url":news_url,
            "content":news_content
        }
        news_dict_list.append(news)

json_data = json.dumps(news_dict_list,indent=4,ensure_ascii=False)
with open("n.json","w",encoding='utf-8') as f:
    f.write(json_data)

